import requests
import json
import os
import pandas as pd
from pandas import DataFrame
from tqdm import tqdm

CAPITAL_TYPE = {
    '1': '政府',
    '2': '社会资本',
    '-1': '空'
}
PROJECT_TYPE = {
    '1':'准备阶段',
    '2':'采购阶段',
    '3':'执行阶段'
}

PERFORM_TYPE={
    '3':'BOT'
}

class GovermentProjReptile:
    def __init__(self, file_path, rid_file_path=None) -> None:
        self.file_path = file_path
        self.init_df = DataFrame(columns=[
            "项目rid",
            "项目name",
            '所在区域',
            '执行阶段',
            '所属行业',
            '发起时间',
            '项目总投资',
            '项目概括',
            '合作时间',
            '运作模式',
            '项目公司/社会资本方',
            '成立时间',
            '注册资金',
            '股东认缴'
        ])
        if rid_file_path and os.path.exists(rid_file_path):
            self.rid_list = self.load_rid(rid_file_path)
        else:
            self.rid_list = self.get_rid_list(init_page=1)


    def run(self):
        self.gen_proj_capital_side(self.rid_list)


    def load_rid(self, file_path):
        with open(file_path, 'r') as f:
            rid_list = [rid.strip() for rid in f.readlines()]
        return rid_list

    def add_rid(self, hits_datas, rid_list):
        for h_d in hits_datas:
            rid_list.append(h_d['proj_rid'])
            with open('rid.txt','a') as f:
                f.write(f"{h_d['proj_rid']}\n")

    def get_rid_list(self, init_page=1):
        # 第一阶段
        rid_list = []
        rid_headers = {
            "Accept": "*/*",
            "Accept-Language": "zh-CN,zh;q=0.9",
            "Content-Type": "application/json",
            "Host":"www.cpppc.org:8082",
            "Origin":"https://www.cpppc.org:8082",
            "Referer":"https://www.cpppc.org:8082/inforpublic/homepage.html",
            "User-Agent": "Mozilla/5.0 (Macintosh; Intel Mac OS X 10_15_7) AppleWebKit/605.1.15 (KHTML, like Gecko) Version/16.0 Safari/605.1.15", 
            "Accept-Encoding": "gzip, deflate, br",
            "Content-Length": "215"
        }

        rid_url = 'https://www.cpppc.org:8082/api/pub/project/search'
        
        payload_data = {
            "name": "",
            "industry": "",
            "min": 0,
            "max": 10000000000000000,
            "pageNumber": init_page,
            "size": 5,
            "status": [
                # "1",
                "2",
                "3" # 执行阶段
            ],
            "level": "",
            "start": "",
            "end": "",
            "dist_province": "",
            "dist_city": "",
            "dist_code": "",
            "nlpVO": {},
            "created_date_order": "desc"
        }

        json_payload_data = json.dumps(payload_data)
        response = requests.post(rid_url, data=json_payload_data, headers=rid_headers).json()
        hits_datas, total_n = response['data']['hits'], response['data']['total']

        self.add_rid(hits_datas, rid_list)

        with tqdm(total=total_n, desc=f'获取工程rid ') as _iter:
            try:
                while len(rid_list) < total_n:
                    payload_data['pageNumber']+=1
                    json_payload_data = json.dumps(payload_data)
                    response = requests.post(rid_url, data=json_payload_data, headers=rid_headers).json()
                    hits_datas, total_n = response['data']['hits'], response['data']['total']
                    self.add_rid(hits_datas, rid_list)
                    _iter.update(len(hits_datas))
            except:
                print(f"{payload_data['pageNumber']} 有问题，自行查看{rid_url}.")
        return rid_list

    def gen_proj_capital_side(self, rid_list):
        run_n = 0
        error_n = 0
        # 第二阶段
        prepare_base = 'https://www.cpppc.org:8082/api/pub/project/detail/'
        perform_base = 'https://www.cpppc.org:8082/api/pub/project/perform-detail/'
        headers = {
                "Accept": "text/html,application/xhtml+xml,application/xml;q=0.9,image/avif,image/webp,image/apng,*/*;q=0.8,application/signed-exchange;v=b3;q=0.9",
                "Accept-Language": "zh-CN,zh;q=0.9",
                "Host":"www.cpppc.org:8082",
                "User-Agent": "Mozilla/5.0 (Macintosh; Intel Mac OS X 10_15_7) AppleWebKit/605.1.15 (KHTML, like Gecko) Version/16.0 Safari/605.1.15", 
                "Accept-Encoding": "gzip, deflate, br",
            }

        exist_rid = None
        if os.path.exists(self.file_path):
            df = pd.read_excel(self.file_path)
            exist_rid = list(df['项目rid'])

        for i in tqdm(range(len(rid_list)), desc=f'下载资料中'):
            _rid = rid_list[i]
            if exist_rid and _rid in exist_rid:
                continue
            try:
                # 准备阶段信息
                prepare_url = prepare_base+_rid
                perform_response = requests.get(prepare_url,headers=headers).json()['data']
                # 项目名称
                pro_name = perform_response['projName']
                # 所在区域
                region = ' - '.join([perform_response['distProvinceName'], perform_response['distCityName'], perform_response['distName']])
                # 执行阶段
                proj_type = PROJECT_TYPE[perform_response['projType']]
                #proj_type = "执行阶段"
                # 所属行业 industryRequiredName + industryOptionalName
                industry = ' - '.join([perform_response['industryRequiredName'], perform_response['industryOptionalName']])
                # 发起时间
                start_time = perform_response['startTime'].split(' ')[0]
                # 项目总投资
                invest_count = perform_response['investCount']
                # 项目概括
                proj_survey = perform_response['projSurvey']
                # 合作时间
                cooperation_term = perform_response['cooperationTerm']
                # 运作模式
                operate_mode = perform_response['operateMode']


                # 执行阶段信息
                perform_url = perform_base+_rid
                perform_response = requests.get(perform_url,headers=headers).json()['data']
                # 项目公司/社会资本方
                organization = perform_response['orginfo'].get('companyName','暂无')
                # 成立时间
                found_time = perform_response['orginfo'].get('foundTime','暂无')
                # 注册资金
                proj_money = perform_response['orginfo'].get('projMoney','暂无')
                # 股东认缴
                equity_list = self.decode_equity(perform_response['orginfo'].get('equityList','暂无'))
                data =[_rid,pro_name,region,proj_type,industry,start_time,invest_count,proj_survey,cooperation_term,operate_mode,organization,found_time,proj_money,equity_list]
                self.write_result(self.file_path, data)
                run_n += 1
            except:
                error_n += 1
                with open('error_logs', 'a') as f:
                    f.write(f'{_rid} error.\n')
        print(f'完成 {run_n}, 错误 {error_n}', )
    def decode_equity(self, equity_list):
        if isinstance(equity_list, dict) and 'ifOpen' in equity_list:
            return equity_list['ifOpen']
        _equity_list = []
        for eq in equity_list:
            _equity_list.append({
                'order_number' : eq['orderNumber'],
                'share_holders_name' : eq.get('shareholdersName', ' '),
                'contribution' : eq.get('contribution', ' '),
                'equity_rate' : eq.get('equityRate', ' '),
                'capital_type' : CAPITAL_TYPE[eq.get('govOrCapital','-1')]
            })
        return _equity_list

    def write_result(self, file_path, data):
        if not os.path.exists(file_path):
            self.init_df.to_excel(file_path, index=False)
        df = pd.read_excel(file_path)
        ds = DataFrame([data], columns = df.columns)
        df = df.append(ds)
        df.to_excel(file_path, index=False)

def revise_df(file_path, save_path):
    df = pd.read_excel(file_path)
    for i in range(len(df)):
        if 'ifOpen' in df.loc[i]['项目公司/社会资本方']:
            df.loc[i, '项目公司/社会资本方'] = df.loc[i, '项目公司/社会资本方'].split(':')[-1][:-1].strip()[1:-1]
        if 'ifOpen' in df.loc[i]['成立时间']:
            df.loc[i,'成立时间'] = df.loc[i, '成立时间'].split(':')[-1][:-1].strip()[1:-1]

    df.to_excel(save_path, index=False)

if __name__ == '__main__':
    worker = GovermentProjReptile('./demo.xls', './rid.txt')
    worker.run()
    revise_df('demo.xls', 'revise.xls')


